# -*- coding: utf-8 -*-
import scrapy

import re
import logging
import urllib.parse
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class DeyangSpider(scrapy.Spider):
    name = 'deyang'
    allowed_domains = ['dyyscx.com']
    start_urls = ['http://www.dyyscx.com/newhouse/houselist.aspx?hou=0-0-0-0-0-0-&page=1']
    project_li_temp_url = 'http://www.dyyscx.com/newhouse/houselist.aspx?hou=0-0-0-0-0-0-&page={}'  # GET 项目列表url
    project_detail_temp_url = 'http://www.dyyscx.com/newhouse/house_base.aspx?id={}'  # GET 项目详情url
    building_li_temp_url = 'http://www.dyyscx.com/newhouse/house_table.aspx?id={}'  # GET  楼栋/许可证 列表url
    room_li_url = 'http://www.dyyscx.com/newhouse/GetBuildTableByAjax.ashx'  # POST 房号列表url

    def parse(self, response):
        """
        获取项目列表
        :param response:
        :return:
        """
        # 获取当前页页数
        try:
            current_page_num = response.meta['current_page_num']
        except:
            current_page_num = 1
        # 获取当前页项目列表
        div_li = response.xpath("//div[@class='newexlmid']")
        if div_li:
            for div in div_li:
                item_eg = FdcEstateGuidItem()
                item_eg['projectName'] = div.xpath("./ul[2]/li[1]/a/text()").extract_first()
                project_main_url = div.xpath("./ul[2]/li[1]/a/@href").extract_first()
                item_eg['projectId'] = self._get_project_id(project_main_url)
                if item_eg['projectId']:
                    item_eg['projectUrl'] = self.project_detail_temp_url.format(item_eg['projectId'])
                    # 获取项目详情
                    headers = {
                        'Referer': urllib.parse.urljoin(response.request.url, project_main_url),
                    }
                    yield scrapy.Request(
                        item_eg['projectUrl'],
                        headers=headers,
                        callback=self.parse_project_detail,
                        meta=dict(item_eg=deepcopy(item_eg), ),
                        priority=6,
                    )
                else:
                    logger.error('第{}页-{} 项目id提取出错'.format(current_page_num, item_eg['projectName'], ))
        else:
            logger.warning(f'第{current_page_num}页 项目列表获取为空')

        # 翻页
        total_num = response.xpath("//span[@id='ts']/text()").extract_first()
        total_page_num = self._get_total_page_num(total_num)
        for next_page_num in range(2, total_page_num + 1):
            headers = {
                'Referer': self.project_li_temp_url.format(current_page_num),
            }
            yield scrapy.Request(
                self.project_li_temp_url.format(next_page_num),
                headers=headers,
                meta=dict(current_page_num=deepcopy(next_page_num), ),
                priority=5,
            )

    def parse_project_detail(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['projectName'] = response.xpath("//ul[@class='newtop']/li[1]/text()").extract_first()
        item_eg['districtName'] = response.xpath(
            "//li[contains(text(),'地区')]/following-sibling::li[1]/text()").extract_first()
        item_eg['projectAddress'] = response.xpath(
            "//li[contains(text(),'项目位置')]/following-sibling::li[1]/text()").extract_first()
        item_eg['salesTel'] = response.xpath(
            "//li[contains(text(),'咨询热线')]/following-sibling::li[1]/text()").extract_first()
        item_eg['openingDate'] = response.xpath(
            "//li[contains(text(),'开盘时间')]/following-sibling::li[1]/text()").extract_first()
        item_eg['handoverDate'] = response.xpath(
            "//li[contains(text(),'入住时间')]/following-sibling::li[1]/text()").extract_first()
        item_eg['developerName'] = response.xpath(
            "//li[contains(text(),'开发商')]/following-sibling::li[1]/text()").extract_first()
        item_eg['planningPermit'] = response.xpath(
            "//li[contains(text(),'规划许可证')]/following-sibling::li[1]/text()").extract_first()
        item_eg['totalHouseholds'] = response.xpath(
            "//li[contains(text(),'总户数')]/following-sibling::li[1]/text()").extract_first()
        item_eg['totalBuildingNum'] = response.xpath(
            "//li[contains(text(),'总栋数')]/following-sibling::li[1]/text()").extract_first()
        item_eg['floorAreaRatio'] = response.xpath(
            "//li[contains(text(),'容积率')]/following-sibling::li[1]/text()").extract_first()
        item_eg['greeningRate'] = response.xpath(
            "//li[contains(text(),'绿化率')]/following-sibling::li[1]/text()").extract_first()
        yield item_eg

        # 获取 楼栋/预售许可证 列表
        yield scrapy.Request(
            self.building_li_temp_url.format(item_eg['projectId']),
            callback=self.parse_building_li,
            meta=dict(item_eg=deepcopy(item_eg), ),
            priority=7,
        )

    def parse_building_li(self, response):
        """
        获取 楼栋/预售许可证 列表
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取 楼栋/预售许可证 分组
        tr_li = response.xpath("//div[@id='BuildTable']/table//tr[position()>1]")
        if tr_li:
            for tr in tr_li:
                item_bd = FdcBuildingItem()
                item_bd['projectName'] = item_eg['projectName']
                item_bd['projectId'] = item_eg['projectId']
                premit_str = tr.xpath("./td[1]/text()").extract_first()
                item_bd['preSalePermit'] = self._get_permit(premit_str)
                item_bd['totalRoomNo'] = tr.xpath("./td[2]/text()").extract_first()
                item_bd['saleableRoomNo'] = tr.xpath("./td[3]/span/text()").extract_first()
                cert_date_str = tr.xpath("./td[4]/text()").extract_first()
                item_bd['certDate'] = self._get_cert_date(cert_date_str)
                item_bd['buildingNo'] = tr.xpath("./td[5]/text()").extract_first()
                building_id_str = tr.xpath("./td[6]/a/@onclick").extract_first()
                item_bd['buildingId'] = self._get_building_id(building_id_str)
                # 获取房号列表
                if item_bd['buildingId']:
                    data = {
                        'itemRecord': str(item_bd['projectId']),
                        'houseCode': str(item_bd['buildingId']),
                    }
                    yield scrapy.FormRequest(
                        self.room_li_url,
                        formdata=data,
                        callback=self.parse_room_li,
                        meta=dict(item_bd=deepcopy(item_bd), ),
                        priority=8,
                    )
                else:
                    logger.error(
                        '{} {}-{} 楼栋id提取出错'.format(response.request.url, item_bd['projectName'], item_bd['buildingNo']))
        else:
            logger.warning('{} {} 楼栋/预售许可证列表获取为空'.format(response.request.url, item_eg['projectName'], ))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        # 单元集合
        unit_set = set()
        # 获取楼层分组
        tr_li = response.xpath("//td[contains(@title,'销售状态')]/..")
        if tr_li:
            for tr in tr_li:
                # 获取楼层
                floor_num_str = tr.xpath("./td[1]/text()").extract_first()
                floor_num = self._get_floor_num(floor_num_str)
                # 获取当前楼层的房号分组
                td_li = tr.xpath("./td[position()>1]")
                if td_li:
                    for td in td_li:
                        item_rm = FdcRoomItem()
                        item_rm['projectName'] = item_bd['projectName']
                        item_rm['projectId'] = item_bd['projectId']
                        item_rm['buildingNo'] = item_bd['buildingNo']
                        item_rm['buildingId'] = item_bd['buildingId']
                        item_rm['roomFloor'] = floor_num
                        room_text_str = td.xpath("./text()").extract_first()
                        item_rm['unitNo'], item_rm['roomNo'] = self._get_unit_no_and_room_no(room_text_str)
                        # 添加单元至集合
                        unit_set.add(item_rm['unitNo']) if item_rm['unitNo'] else False
                        room_title_str = td.xpath("./@title").extract_first()
                        room_title_dict = self._deal_room_title_str_to_dict(room_title_str, deepcopy(item_rm))
                        if room_title_dict:
                            item_rm['roomUse'] = room_title_dict.get('物业类别', None)
                            item_rm['saleStatus'] = room_title_dict.get('销售状态', None)
                            item_rm['buildingArea'] = room_title_dict.get('建筑面积', None)
                            item_rm['recordTotalPrice'] = room_title_dict.get('总价', None)
                            yield item_rm
                else:
                    logger.warning(
                        '{}-{}-{} 该楼层房号分组获取为空'.format(item_bd['projectName'], item_bd['buildingNo'], floor_num))
        else:
            logger.warning('{}-{} 楼层分组获取为空'.format(item_bd['projectName'], item_bd['buildingNo']))

        # 楼栋字典添加单元列表字段
        unit_list = list(unit_set) if unit_set else ['1', ]
        item_bd['unitLi'] = unit_list
        yield item_bd

    @staticmethod
    def _get_total_page_num(_str: str) -> int:
        """
        提取总页数
        :param _str:
        :return:
        """
        try:
            total_num = int(_str)
            ret = total_num // 10 + 1 if total_num % 10 else total_num // 10
            assert ret
        except:
            logger.error('总页数提取出错')
            return 37  # 提取出错时，设置一个默认值，防止程序无法继续爬取
        else:
            return ret

    @staticmethod
    def _get_project_id(_str: str) -> str or None:
        """
        提取项目id
        :param _str:
        :return:
        """
        try:
            ret = _str.split('=')[1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_permit(_str) -> str or None:
        """
        提取预售许可证
        :param _str:
        :return:
        """
        try:
            ret = _str.split(':')[1]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_cert_date(_str) -> str or None:
        """
        提取发证日期
        :param _str:
        :return:
        """
        regex = re.compile(r'\d+/\d+/\d+?')
        try:
            ret = regex.findall(_str)[0]
            assert ret
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_building_id(_str) -> str or None:
        """
        提取楼栋id
        :param _str:
        :return:
        """
        regex = re.compile(r',\'(.*)\'')
        try:
            ret = regex.findall(_str)[0]
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_floor_num(_str) -> str or None:
        """
        提取楼层num
        :param _str:
        :return:
        """
        regex = re.compile(r'-*\d+')
        try:
            ret = regex.findall(_str)[0]
        except:
            return
        else:
            return ret

    @staticmethod
    def _get_unit_no_and_room_no(_str) -> tuple:
        """
        提取单元号和房号
        :param _str:
        :return:
        """
        try:
            ret_li = [i.strip() for i in _str.split('单元')]
            assert len(ret_li) and len(ret_li) <= 2
            # 有单元号
            if len(ret_li) == 2:
                unit_no = ret_li[0]
                room_no = ret_li[1]
            # 无单元号
            else:
                unit_no = None
                room_no = ret_li[0]
        except:
            return None, None
        else:
            return unit_no, room_no

    @staticmethod
    def _deal_room_title_str_to_dict(_str, item) -> dict:
        """
        将房号信息字符串反序列化为字典
        :param _str:
        :param item:
        :return:
        """
        try:
            # 字符串转换为列表，并去除空字符串
            ret_li = [i for i in _str.replace('：', ':').split(' ') if i]
            # 列表转换为字典
            ret_dict = {i.split(':')[0]: i.split(':')[1] for i in ret_li}
        except Exception as e:
            logger.error(
                '{}-{}-{}-{} 房号信息反序列化为字典出错,error_msg:{}'.format(item['projectName'], item['buildingNo'],
                                                                item['roomFloor'],
                                                                item['roomNo'], e))
            del item
            return dict()
        else:
            return ret_dict
